{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "executionInfo": {
     "elapsed": 6862,
     "status": "ok",
     "timestamp": 1650010812842,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "98nP9Uh9GUTL"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARoAAAESCAYAAAAi4BrXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAQU0lEQVR4nO3dcWhTd7/H8U/SYVs1DdOpXbFq7QQZRcujtg5hq6zXCnJB/1D/8AFbpMxRxeIfane5lv3xEHFlE4qoDKayTRQELY4pKzIrA0WxE6ajQp/NWdq1RseSrLBUknP/8DEzs9bU6zfntL5fcJCcJD1fxbw55xdNfI7jOAIAQ363BwAw/hEaAOYIDQBzhAaAOUIDwByhAWCO0AAw94rbA4wkmUyqr69PgUBAPp/P7XEA/I3jOIrFYioqKpLf//TzFk+Hpq+vT8XFxW6PAeAZenp6NHPmzKfe7+nQBAIBSQ9/EwUFBS5PA+DvotGoiouLU6/Vp/F0aB5dLhUUFBAawMOetbTBYjAAc4QGgDlCA8AcoQFgjtAAMEdoAJgjNADMERoA5ggNAHOEBoA5QgPAHKEBYI7QADBHaACYIzQAzBEaAOYIDQBzWQvNnj175PP51NjYmK1DAvCIrITm6tWrOnTokBYsWJCNwwHwGPPQ/PHHH9qwYYM+/fRTvfrqqyM+Nh6PKxqNpm0Axj7z0DQ0NGjVqlWqrq5+5mNDoZCCwWBq46tWgPHBNDTHjx9XZ2enQqFQRo9vampSJBJJbT09PZbjAcgSs69b6enp0bZt29Te3q68vLyMnpObm6vc3FyrkQC4xOc4jmPxg0+fPq01a9YoJycntS+RSMjn88nv9ysej6fdN5xoNKpgMKhIJML3OgEelOlr1OyM5t1339UPP/yQtq+urk7z58/Xzp07nxkZAOOHWWgCgYDKysrS9k2aNElTp059Yj+A8Y1/GQzAXFa/e/vChQvZPBwAj+CMBoA5QgPAHKEBYI7QADBHaACYIzQAzBEaAOYIDQBzhAaAOUIDwByhAWCO0AAwR2gAmCM0AMwRGgDmCA0Ac4QGgDlCA8AcoQFgjtAAMEdoAJgjNADMERoA5ggNAHOEBoA5QgPAHKEBYI7QADBHaACYIzQAzBEaAOYIDQBzhAaAOUIDwByhAWDONDShUEhLlixRIBDQ9OnTtXr1at26dcvykAA8yDQ0HR0damho0OXLl9Xe3q4HDx5oxYoVGhwctDwsAI/xOY7jZOtg4XBY06dPV0dHh95+++1nPj4ajSoYDCoSiaigoCALEwIYjUxfo69kcSZFIhFJ0pQpU4a9Px6PKx6Pp25Ho9GszAXAVtYWg5PJpBobG7Vs2TKVlZUN+5hQKKRgMJjaiouLszUeAENZu3R6//33dfbsWX333XeaOXPmsI8Z7oymuLiYSyfAozx16bRlyxZ99dVXunjx4lMjI0m5ubnKzc3NxkgAssg0NI7jaOvWrTp16pQuXLigkpISy8MB8CjT0DQ0NOjYsWNqa2tTIBBQf3+/JCkYDCo/P9/y0AA8xHSNxufzDbv/8OHDqq2tfebzeXsb8DZPrNFk8Z/oAPAw/q8TAHOEBoA5QgPAHKEBYI7QADBHaACYIzQAzBEaAOYIDQBzhAaAOUIDwByhAWCO0AAwR2gAmCM0AMwRGgDmCA0Ac4QGgDlCA8AcoQFgjtAAMEdoAJgjNADMERoA5ggNAHOEBoA5QgPAHKEBYI7QADBHaACYIzQAzBEaAOYIDQBzhAaAOUIDwJx5aPbv3685c+YoLy9PlZWVunLlivUhAXiMaWhOnDih7du3q7m5WZ2dnVq4cKFqamp09+5dy8MC8BjT0Hz88ceqr69XXV2d3nzzTR08eFATJ07UZ599ZnlYYMxJJB1d+vd9tV3v1aV/31ci6bg90gv1itUPHhoa0rVr19TU1JTa5/f7VV1drUuXLg37nHg8rng8nrodjUatxgM849yNX/XhmR/1a+TP1L7Xg3lq/u83tbLsdRcne3HMzmju3bunRCKhGTNmpO2fMWOG+vv7h31OKBRSMBhMbcXFxVbjAZ5w7savev+LzrTISFJ/5E+9/0Wnzt341aXJXixPvevU1NSkSCSS2np6etweCTCTSDr68MyPGu4i6dG+D8/8OC4uo8wunV577TXl5ORoYGAgbf/AwIAKCwuHfU5ubq5yc3OtRgI85crPvz1xJvM4R9KvkT915eff9Fbp1OwNZsDsjGbChAlatGiRzp8/n9qXTCZ1/vx5vfXWW1aHBcaMu7GnR+Z5HudlZmc0krR9+3Zt3LhRixcvVkVFhfbt26fBwUHV1dVZHhYYE6YH8l7o47zMNDTr169XOBzW7t271d/fr/Lycp07d+6JBWLgZVRRMkWvB/PUH/lz2HUan6TCYJ4qSqZke7QXzuc4jmdXmqLRqILBoCKRiAoKCtweB3jhHr3rJCktNr7//Hrgn//w9Fvcmb5GPfWuE/CyWVn2ug788x8qDKZfHhUG8zwfmdEwvXQC8Gwry17Xf71ZqCs//6a7sT81PfDwcinH73v2k8cIQgN4QI7fN+bfwh4Jl04AzBEaAOYIDQBzhAaAOUIDwByhAWCO0AAwR2gAmCM0AMwRGgDmCA0Ac4QGgDlCA8AcoQFgjtAAMEdoAJgjNADMERoA5ggNAHOEBoA5QgPAHKEBYI7QADBHaACYIzQAzBEaAOYIDQBzhAaAOUIDwByhAWCO0AAwR2gAmDMJze3bt7Vp0yaVlJQoPz9fpaWlam5u1tDQkMXhAHjcKxY/tKurS8lkUocOHdIbb7yhGzduqL6+XoODg2ppabE4JAAP8zmO42TjQB999JEOHDign376KePnRKNRBYNBRSIRFRQUGE4H4Hlk+ho1OaMZTiQS0ZQpU0Z8TDweVzweT92ORqPWYwHIgqwsBnd3d6u1tVXvvffeiI8LhUIKBoOprbi4OBvjATA2qtDs2rVLPp9vxK2rqyvtOb29vVq5cqXWrl2r+vr6EX9+U1OTIpFIauvp6Rn97wiA54xqjSYcDuv+/fsjPmbu3LmaMGGCJKmvr09VVVVaunSpjhw5Ir9/dCdQrNEA3mayRjNt2jRNmzYto8f29vZq+fLlWrRokQ4fPjzqyAAYP0wWg3t7e1VVVaXZs2erpaVF4XA4dV9hYaHFIQF4mElo2tvb1d3dre7ubs2cOTPtviy9mw7AQ0yuZ2pra+U4zrAbgJcPCycAzBEaAOYIDQBzhAaAOUIDwByhAWCO0AAwR2gAmCM02fBtSOrYO/x9HXsf3g+MY4QmG/w50rf/ejI2HXsf7vfnuDMXkCVZ+4S9l9o7Ox7++u2//rr9KDLL/+ev+4FxitBky+OxufiRlBgiMnhpcOmUTe/skHImPIxMzgQig5cGocmmjr1/RSYx9PQFYmCcITTZ8viazP+GH/463AIxMA6xRpMNwy38DrdADIxThCYbkonhF34f3U4msj8TkEVZ+6bK58G3IADelulrlDUaAOYIDQBzhAaAOUIDwByhAWCO0AAwR2gAmCM0AMwRGgDmCA0Ac4QGgDlCA8AcoQFgjtAAMEdoAJgjNADMERoA5sxDE4/HVV5eLp/Pp+vXr1sfDoAHmYdmx44dKioqsj4MAA8zDc3Zs2f1zTffqKWlxfIwADzO7FsQBgYGVF9fr9OnT2vixIkZPScejysej6duR6NRq/EAZJHJGY3jOKqtrdXmzZu1ePHijJ8XCoUUDAZTW3FxscV4ALJsVKHZtWuXfD7fiFtXV5daW1sVi8XU1NQ0qmGampoUiURSW09Pz6ieD8CbRvW9TuFwWPfv3x/xMXPnztW6det05swZ+Xy+1P5EIqGcnBxt2LBBR48ezeh4fK8T4G2ZvkZNvkDuzp07aesrfX19qqmp0cmTJ1VZWamZM2dm9HMIDeBtmb5GTRaDZ82alXZ78uTJkqTS0tKMIwNg/OBfBgMwZ/b29uPmzJkjD3/FNwBjnNEAMEdoAJgjNADMERoA5ggNAHOEBoA5QgPAHKEBYI7QADBHaACYIzQAzBEaAOYIDQBzhAaAOUIDwFxWPo/meT36DBu+dgXwpkevzWd93pSnQxOLxSSJr10BPC4WiykYDD71fpMPJ39Rksmk+vr6FAgE0r5R4UWIRqMqLi5WT0/PmPvg87E6+1idWxq7s1vP7TiOYrGYioqK5Pc/fSXG02c0fr/f/MPMCwoKxtRfnMeN1dnH6tzS2J3dcu6RzmQeYTEYgDlCA8DcSxua3NxcNTc3Kzc31+1RRm2szj5W55bG7uxemdvTi8EAxoeX9owGQPYQGgDmCA0Ac4QGgDlCA8AcofmbeDyu8vJy+Xw+Xb9+3e1xRnT79m1t2rRJJSUlys/PV2lpqZqbmzU0NOT2aMPav3+/5syZo7y8PFVWVurKlStujzSiUCikJUuWKBAIaPr06Vq9erVu3brl9ljPZc+ePfL5fGpsbHTl+ITmb3bs2KGioiK3x8hIV1eXksmkDh06pJs3b+qTTz7RwYMH9cEHH7g92hNOnDih7du3q7m5WZ2dnVq4cKFqamp09+5dt0d7qo6ODjU0NOjy5ctqb2/XgwcPtGLFCg0ODro92qhcvXpVhw4d0oIFC9wbwkHK119/7cyfP9+5efOmI8n5/vvv3R5p1Pbu3euUlJS4PcYTKioqnIaGhtTtRCLhFBUVOaFQyMWpRufu3buOJKejo8PtUTIWi8WcefPmOe3t7c4777zjbNu2zZU5OKP5j4GBAdXX1+vzzz/XxIkT3R7nuUUiEU2ZMsXtMdIMDQ3p2rVrqq6uTu3z+/2qrq7WpUuXXJxsdCKRiCR57s93JA0NDVq1alXan70bPP2/t7PFcRzV1tZq8+bNWrx4sW7fvu32SM+lu7tbra2tamlpcXuUNPfu3VMikdCMGTPS9s+YMUNdXV0uTTU6yWRSjY2NWrZsmcrKytweJyPHjx9XZ2enrl696vYo43uNZteuXfL5fCNuXV1dam1tVSwWU1NTk9sjS8p87sf19vZq5cqVWrt2rerr612afPxqaGjQjRs3dPz4cbdHyUhPT4+2bdumL7/8Unl5eW6PM77/r1M4HNb9+/dHfMzcuXO1bt06nTlzJu3DtRKJhHJycrRhwwYdPXrUetQ0mc49YcIESVJfX5+qqqq0dOlSHTlyZMQPIHLD0NCQJk6cqJMnT2r16tWp/Rs3btTvv/+utrY294bLwJYtW9TW1qaLFy+qpKTE7XEycvr0aa1Zs0Y5OTmpfYlEQj6fT36/X/F4PO0+a+M6NJm6c+dO2ucS9/X1qaamRidPnlRlZaX5h2/9f/T29mr58uVatGiRvvjii6z+5RmNyspKVVRUqLW1VdLDS5FZs2Zpy5Yt2rVrl8vTDc9xHG3dulWnTp3ShQsXNG/ePLdHylgsFtMvv/yStq+urk7z58/Xzp07s375xxqNpFmzZqXdnjx5siSptLTU85GpqqrS7Nmz1dLSonA4nLqvsLDQxcmetH37dm3cuFGLFy9WRUWF9u3bp8HBQdXV1bk92lM1NDTo2LFjamtrUyAQUH9/v6SHnyiXn5/v8nQjCwQCT8Rk0qRJmjp1qitrTIRmDGtvb1d3d7e6u7ufCKLXTlTXr1+vcDis3bt3q7+/X+Xl5Tp37twTC8RecuDAAUlSVVVV2v7Dhw+rtrY2+wONYVw6ATDnrVVDAOMSoQFgjtAAMEdoAJgjNADMERoA5ggNAHOEBoA5QgPAHKEBYI7QADD3f3s6GgIZCNvhAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper:\n",
    "\n",
    "    def reset(self):\n",
    "        #前两个数是起点,后两个数是终点\n",
    "        self.state = np.random.uniform(size=4, low=-5, high=5)\n",
    "        self.step_n = 0\n",
    "        return self.state.tolist()\n",
    "\n",
    "    def step(self, action):\n",
    "        action = np.array(action).reshape(2)\n",
    "\n",
    "        #裁剪动作范围\n",
    "        action = action.clip(min=-1, max=1)\n",
    "\n",
    "        #执行动作\n",
    "        self.state[:2] += action\n",
    "\n",
    "        #规范状态空间\n",
    "        self.state[:2] = self.state[:2].clip(min=-5, max=5)\n",
    "\n",
    "        #求距离终点的距离\n",
    "        dist = np.linalg.norm(self.state[:2] - self.state[2:], ord=2)\n",
    "\n",
    "        #判断到达终点\n",
    "        reward = -1.0\n",
    "        over = False\n",
    "        if dist < 0.1:\n",
    "            reward = 1.0\n",
    "            over = True\n",
    "\n",
    "        #限制最大步数\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 50:\n",
    "            over = True\n",
    "\n",
    "        return self.state.tolist(), reward, over\n",
    "\n",
    "    def show(self):\n",
    "        from matplotlib import pyplot as plt\n",
    "        plt.figure(figsize=(3, 3))\n",
    "        plt.xlim(-5, 5)\n",
    "        plt.ylim(-5, 5)\n",
    "\n",
    "        plt.plot(*self.state[:2], 'o')\n",
    "        plt.plot(*self.state[2:], 'x')\n",
    "\n",
    "        plt.show()\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()\n",
    "\n",
    "env.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "executionInfo": {
     "elapsed": 2,
     "status": "ok",
     "timestamp": 1650010819329,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "bxiqOl_vGUTR"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<ddpg.DDPG at 0x7f5b280527a0>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from ddpg import DDPG\n",
    "import torch\n",
    "\n",
    "#定义算法模型\n",
    "ddpg = DDPG()\n",
    "\n",
    "ddpg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-50.0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "#玩一局游戏并记录数据\n",
    "def play(show=False):\n",
    "    data = []\n",
    "    reward_sum = 0\n",
    "\n",
    "    state = env.reset()\n",
    "    over = False\n",
    "    while not over:\n",
    "        action = ddpg.model_action(torch.FloatTensor(state).reshape(\n",
    "            1, 4)).reshape(2)\n",
    "\n",
    "        #给动作添加噪声,增加探索\n",
    "        action += torch.randn(2) * 0.1\n",
    "        action = action.tolist()\n",
    "\n",
    "        next_state, reward, over = env.step(action)\n",
    "\n",
    "        data.append((state, action, reward, next_state, over))\n",
    "        reward_sum += reward\n",
    "\n",
    "        state = next_state\n",
    "\n",
    "        if show:\n",
    "            display.clear_output(wait=True)\n",
    "            env.show()\n",
    "\n",
    "    return data, reward_sum\n",
    "\n",
    "\n",
    "play()[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([2.5521624730177876,\n",
       "  2.3458911568199117,\n",
       "  2.7167221130081174,\n",
       "  2.4791729390893895],\n",
       " [0.053365983068943024, 0.025464773178100586],\n",
       " -1.0,\n",
       " [2.6055284560867307,\n",
       "  2.3713559299980123,\n",
       "  2.7167221130081174,\n",
       "  2.4791729390893895],\n",
       " False)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#从一局游戏中取一条伪终点的数据\n",
    "def get_fake_goal_data(data_game, step):\n",
    "    #取出step的数据\n",
    "    state, action, reward, next_state, over = data_game[step]\n",
    "\n",
    "    #随机step后面的某一步数据\n",
    "    step = random.randint(step + 1, len(data_game) - 1)\n",
    "    fake_goal_state = data_game[step][0]\n",
    "\n",
    "    #以伪终点构建新的state\n",
    "    state[2:] = fake_goal_state[:2]\n",
    "    next_state[2:] = fake_goal_state[:2]\n",
    "\n",
    "    #求距离终点的距离\n",
    "    dist = [next_state[0] - next_state[2], next_state[1] - next_state[3]]\n",
    "    dist = np.linalg.norm(dist, ord=2)\n",
    "\n",
    "    #重新计算reward和over\n",
    "    reward = -1.0\n",
    "    over = False\n",
    "    if dist < 0.1:\n",
    "        reward = 1.0\n",
    "        over = True\n",
    "\n",
    "    #返回作为伪终点数据\n",
    "    return state, action, reward, next_state, over\n",
    "\n",
    "\n",
    "get_fake_goal_data(play()[0], 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200,\n",
       " ([3.7959772773839706, -2.729194208223278, 5.0, 0.2494128652029688],\n",
       "  [0.041604362428188324, 0.07898694276809692],\n",
       "  -1.0,\n",
       "  [3.837581639812159, -2.650207265455181, 5.0, 0.2494128652029688],\n",
       "  False))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#数据池\n",
    "class Pool:\n",
    "\n",
    "    def __init__(self):\n",
    "        self.pool = []\n",
    "\n",
    "    def __len__(self):\n",
    "        return sum(len(i) for i in self.pool)\n",
    "\n",
    "    def __getitem__(self, i):\n",
    "        return self.pool[i]\n",
    "\n",
    "    #更新动作池\n",
    "    def update(self):\n",
    "        #每次更新不少于N条新数据\n",
    "        old_len = len(self)\n",
    "        while len(self) - old_len < 200:\n",
    "            self.pool.append(play()[0])\n",
    "\n",
    "        #只保留最新的N条数据\n",
    "        while len(self) > 2_0000:\n",
    "            self.pool = self.pool[1:]\n",
    "\n",
    "    #获取一批数据样本\n",
    "    def sample(self):\n",
    "        data = []\n",
    "        for _ in range(64):\n",
    "            #随机一局游戏\n",
    "            data_game = random.choice(self.pool)\n",
    "\n",
    "            #随机取游戏中的一步数据\n",
    "            step = random.randint(0, len(data_game) - 1)\n",
    "            data_step = data_game[step]\n",
    "\n",
    "            #随机替换为伪终点数据\n",
    "            if step < len(data_game) - 1 and random.random() < 0.8:\n",
    "                data_step = get_fake_goal_data(data_game, step)\n",
    "\n",
    "            data.append(data_step)\n",
    "\n",
    "        state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 4)\n",
    "        action = torch.FloatTensor([i[1] for i in data]).reshape(-1, 2)\n",
    "        reward = torch.FloatTensor([i[2] for i in data]).reshape(-1, 1)\n",
    "        next_state = torch.FloatTensor([i[3] for i in data]).reshape(-1, 4)\n",
    "        over = torch.LongTensor([i[4] for i in data]).reshape(-1, 1)\n",
    "\n",
    "        return state, action, reward, next_state, over\n",
    "\n",
    "\n",
    "pool = Pool()\n",
    "pool.update()\n",
    "pool.sample()\n",
    "\n",
    "len(pool), pool[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 400 -50.0\n",
      "100 19953 -17.9\n",
      "200 19960 -4.8\n",
      "300 19995 -5.4\n",
      "400 19998 -2.9\n",
      "500 19995 -5.15\n",
      "600 19999 -5.45\n",
      "700 19996 -3.85\n",
      "800 19999 -5.0\n",
      "900 19998 -4.35\n"
     ]
    }
   ],
   "source": [
    "#训练\n",
    "def train():\n",
    "    #共更新N轮数据\n",
    "    for epoch in range(1000):\n",
    "        pool.update()\n",
    "\n",
    "        #每次更新数据后,训练N次\n",
    "        for i in range(200):\n",
    "\n",
    "            #采样N条数据\n",
    "            state, action, reward, next_state, over = pool.sample()\n",
    "\n",
    "            #训练模型\n",
    "            ddpg.train_action(state)\n",
    "            ddpg.train_value(state, action, reward, next_state, over)\n",
    "            ddpg.soft_update()\n",
    "\n",
    "        if epoch % 100 == 0:\n",
    "            test_result = sum([play()[-1] for _ in range(20)]) / 20\n",
    "            print(epoch, len(pool), test_result)\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARoAAAESCAYAAAAi4BrXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAQWUlEQVR4nO3db2hUh5rH8d9MrJOok6Fa/zQ4akxlRUTDGhO7QhtpUEEuaKn6wgUT3FBLtIYU1BTWsC92R9q0FYKodKm6bUWXUBVLlQZZIwVFMRWql8gNvda5SROjl85MA07szNkXtrNNjXHi+syZxO8Hzos5cybniThfzjkzmfE4juMIAAx53R4AwOhHaACYIzQAzBEaAOYIDQBzhAaAOUIDwNwYtwcYSjKZVFdXl/x+vzwej9vjAPgDx3EUi8VUUFAgr/fRxy1ZHZquri4Fg0G3xwDwGOFwWNOnT3/k/VkdGr/fL+nBL5Gfn+/yNAD+KBqNKhgMpp6rj5LVofntdCk/P5/QAFnscZc2uBgMwByhAWCO0AAwR2gAmCM0AMwRGgDmCA0Ac4QGgDlCA8AcoQFgjtAAMEdoAJgjNADMERoA5ggNAHOEBoA5QgPAXMZCs3v3bnk8HtXW1mZqlwCyREZCc/nyZR04cEALFizIxO4AZBnz0Pz888/asGGDPv74Yz3//PNDbhuPxxWNRgcsAEY+89DU1NRo1apVqqioeOy2oVBIgUAgtfBVK8DoYBqao0ePqq2tTaFQKK3t6+vrFYlEUks4HLYcD0CGmH3dSjgc1rZt29TS0qLc3Ny0HuPz+eTz+axGAuASj+M4jsUPPnHihNasWaOcnJzUukQiIY/HI6/Xq3g8PuC+wUSjUQUCAUUiEb7XCchC6T5HzY5oXnvtNX333XcD1lVVVWnu3LnasWPHYyMDYPQwC43f79f8+fMHrBs/frwmTZr00HoAoxvvDAZgLqPfvX3u3LlM7g5AluCIBoA5QgPAHKEBYI7QADBHaACYIzQAzBEaAOYIDQBzhAaAOUIDwByhAWCO0AAwR2gAmCM0AMwRGgDmCA0Ac4QGgDlCA8AcoQFgjtAAMEdoAJgjNADMERoA5ggNAHOEBoA5QgPAHKEBYI7QADBHaACYIzQAzBEaAOYIDQBzhAaAOUIDwByhAWDONDShUEiLFy+W3+/XlClTtHr1at24ccNylwCykGloWltbVVNTo4sXL6qlpUX379/X8uXL1dfXZ7lbAFnG4ziOk6md9fb2asqUKWptbdUrr7zy2O2j0agCgYAikYjy8/MzMCGA4Uj3OTomgzMpEolIkiZOnDjo/fF4XPF4PHU7Go1mZC4AtjJ2MTiZTKq2tlZLly7V/PnzB90mFAopEAiklmAwmKnxABjK2KnTW2+9pdOnT+ubb77R9OnTB91msCOaYDDIqROQpbLq1GnLli368ssvdf78+UdGRpJ8Pp98Pl8mRgKQQaahcRxHW7du1fHjx3Xu3DkVFhZa7g5AljINTU1NjY4cOaKTJ0/K7/eru7tbkhQIBJSXl2e5awBZxPQajcfjGXT9wYMHVVlZ+djH8/I2kN2y4hpNBt+iAyCL8bdOAMwRGgDmCA0Ac4QGgDlCA8AcoQFgjtAAMEdoAJgjNADMERoA5ggNAHOEBoA5QgPAHKEBYI7QADBHaACYIzQAzBEaAOYIDQBzhAaAOUIDwByhAWAuI1+J+6xJJB1d+uvfdTt2T1P8uSotnKgc7+DfcQU8CwjNU3bm2o/62/FditxLqinxuiTpxUCuGv40Tyvnvyi1viclE9KyepcnBTKHU6en6My1H/XWZ22K3EvqneeatTXnC0lSd+Se3vqsTX/573+V/uffJW+Oy5MCmcURzVOSSDr6t1N/liOljmTeea5Z+vX22zlfaM6fm5Usf1feV7e7OCmQeYTmKbn017/rx8i91O3fx2bLmBPyeX7RB/ff0D8F/0UvuzUk4BJOnZ6S27F7D61rSryuuDNGPs8vijtj1JR4fdDtgNGO0DwlU/y5D63bmvNFKjI+zy/amvPFoNsBox2heUpKCyfqxUCufnsRe2vOF3rnuWZ9cP8N/UP8v/Th/Tf0znPNKgv/p6tzAm4gNE9Jjtejhj/NkyS9/bvINCVel0cPTqP+Mu9tec/9x4OXuIFnCBeDn6KV81/Uvn/+R/3t+Al9cO+N1AXhab++j2bO/FVSq//B+2iAZ4jHcRzH7SEeJRqNKhAIKBKJKD8/3+1x0sY7g/GsSPc5yhGNgRyvRy8XTXJ7DCBrcI0GgDnz0Ozdu1ezZs1Sbm6uysrKdOnSJetdAsgypqE5duyY6urq1NDQoLa2Ni1cuFArVqzQ7du3LXcLIMuYhubDDz9UdXW1qqqqNG/ePO3fv1/jxo3TJ598YrlbAFnGLDT9/f26cuWKKioq/m9nXq8qKip04cKFQR8Tj8cVjUYHLABGPrPQ3LlzR4lEQlOnTh2wfurUqeru7h70MaFQSIFAILUEg0Gr8QBkUFa96lRfX69IJJJawuGw2yMBeArM3kfzwgsvKCcnRz09PQPW9/T0aNq0aYM+xufzyefzWY0EwCVmRzRjx47VokWLdPbs2dS6ZDKps2fP6uWX+UQW4Fli+s7guro6bdy4USUlJSotLdWePXvU19enqqoqy90CyDKmoVm/fr16e3u1a9cudXd3q7i4WGfOnHnoAjGA0Y0/qgTwxNJ9jmbVq04ARidCA8AcoQFgjtAAMEdoAJgjNADMERoA5ggNAHOEBoA5QgPAHKEBYI7QADBHaACYIzQAzBEaAOYIDQBzhAaAOUIDwByhAWCO0AAwR2gAmCM0AMwRGgDmCA0Ac4QGgDlCA8AcoQFgjtAAMEdoAJgjNADMERoA5ggNAHOEBoA5QgPAHKEBYM4kNDdv3tSmTZtUWFiovLw8FRUVqaGhQf39/Ra7A5Dlxlj80Pb2diWTSR04cEAvvfSSrl27purqavX19amxsdFilwCymMdxHCcTO3r//fe1b98+ff/992k/JhqNKhAIKBKJKD8/33A6AE8i3eeoyRHNYCKRiCZOnDjkNvF4XPF4PHU7Go1ajwUgAzJyMbijo0NNTU168803h9wuFAopEAiklmAwmInxABgbVmh27twpj8cz5NLe3j7gMZ2dnVq5cqXWrl2r6urqIX9+fX29IpFIagmHw8P/jQBknWFdo+nt7dXdu3eH3Gb27NkaO3asJKmrq0vl5eVasmSJDh06JK93eAdQXKMBspvJNZrJkydr8uTJaW3b2dmpZcuWadGiRTp48OCwIwNg9DC5GNzZ2any8nLNnDlTjY2N6u3tTd03bdo0i10CyGImoWlpaVFHR4c6Ojo0ffr0Afdl6NV0AFnE5HymsrJSjuMMugB49nDhBIA5QgPAHKEBYI7QADBHaACYIzQAzBEaAOYIDQBzhAaAOUIDwByhAWCO0AAwR2gAmCM0AMwRGgDmCA0Ac4QGgDlCA8AcoQFgjtAAMEdoAJgjNADMERoA5ggNAHOEBoA5QgPAHKEBYI7QADBHaACYIzQAzBEaAOYIDQBzhAaAOUIDwByhAWDOPDTxeFzFxcXyeDy6evWq9e4AZCHz0Gzfvl0FBQXWuwGQxUxDc/r0aX399ddqbGy03A2ALDfG6gf39PSourpaJ06c0Lhx49J6TDweVzweT92ORqNW4wHIIJMjGsdxVFlZqc2bN6ukpCTtx4VCIQUCgdQSDAYtxgOQYcMKzc6dO+XxeIZc2tvb1dTUpFgspvr6+mENU19fr0gkklrC4fCwHg8gO3kcx3HS3bi3t1d3794dcpvZs2dr3bp1OnXqlDweT2p9IpFQTk6ONmzYoMOHD6e1v2g0qkAgoEgkovz8/HTHBJAh6T5HhxWadN26dWvA9ZWuri6tWLFCzc3NKisr0/Tp09P6OYQGyG7pPkdNLgbPmDFjwO0JEyZIkoqKitKODIDRg3cGAzBn9vL2782aNUsGZ2gARgiOaACYIzQAzBEaAOYIDQBzhAaAOUIDwByhAWCO0AAwR2gAmCM0AMwRGgDmCA0Ac4QGgDlCA8AcoQFgLiOfR/OkfvsMG752BchOvz03H/d5U1kdmlgsJkl87QqQ5WKxmAKBwCPvN/lw8qclmUyqq6tLfr9/wDcqPA3RaFTBYFDhcHjEffD5SJ19pM4tjdzZred2HEexWEwFBQXyeh99JSarj2i8Xq/5h5nn5+ePqP84vzdSZx+pc0sjd3bLuYc6kvkNF4MBmCM0AMw9s6Hx+XxqaGiQz+dze5RhG6mzj9S5pZE7e7bMndUXgwGMDs/sEQ2AzCE0AMwRGgDmCA0Ac4QGgDlC8wfxeFzFxcXyeDy6evWq2+MM6ebNm9q0aZMKCwuVl5enoqIiNTQ0qL+/3+3RBrV3717NmjVLubm5Kisr06VLl9weaUihUEiLFy+W3+/XlClTtHr1at24ccPtsZ7I7t275fF4VFtb68r+Cc0fbN++XQUFBW6PkZb29nYlk0kdOHBA169f10cffaT9+/fr3XffdXu0hxw7dkx1dXVqaGhQW1ubFi5cqBUrVuj27dtuj/ZIra2tqqmp0cWLF9XS0qL79+9r+fLl6uvrc3u0Ybl8+bIOHDigBQsWuDeEg5SvvvrKmTt3rnP9+nVHkvPtt9+6PdKwvffee05hYaHbYzyktLTUqampSd1OJBJOQUGBEwqFXJxqeG7fvu1IclpbW90eJW2xWMyZM2eO09LS4rz66qvOtm3bXJmDI5pf9fT0qLq6Wp9++qnGjRvn9jhPLBKJaOLEiW6PMUB/f7+uXLmiioqK1Dqv16uKigpduHDBxcmGJxKJSFLW/fsOpaamRqtWrRrwb++GrP7r7UxxHEeVlZXavHmzSkpKdPPmTbdHeiIdHR1qampSY2Oj26MMcOfOHSUSCU2dOnXA+qlTp6q9vd2lqYYnmUyqtrZWS5cu1fz5890eJy1Hjx5VW1ubLl++7PYoo/sazc6dO+XxeIZc2tvb1dTUpFgspvr6erdHlpT+3L/X2dmplStXau3ataqurnZp8tGrpqZG165d09GjR90eJS3hcFjbtm3T559/rtzcXLfHGd1/69Tb26u7d+8Ouc3s2bO1bt06nTp1asCHayUSCeXk5GjDhg06fPiw9agDpDv32LFjJUldXV0qLy/XkiVLdOjQoSE/gMgN/f39GjdunJqbm7V69erU+o0bN+qnn37SyZMn3RsuDVu2bNHJkyd1/vx5FRYWuj1OWk6cOKE1a9YoJycntS6RSMjj8cjr9Soejw+4z9qoDk26bt26NeBzibu6urRixQo1NzerrKzM/MO3/j86Ozu1bNkyLVq0SJ999llG//MMR1lZmUpLS9XU1CTpwanIjBkztGXLFu3cudPl6QbnOI62bt2q48eP69y5c5ozZ47bI6UtFovphx9+GLCuqqpKc+fO1Y4dOzJ++sc1GkkzZswYcHvChAmSpKKioqyPTHl5uWbOnKnGxkb19vam7ps2bZqLkz2srq5OGzduVElJiUpLS7Vnzx719fWpqqrK7dEeqaamRkeOHNHJkyfl9/vV3d0t6cEnyuXl5bk83dD8fv9DMRk/frwmTZrkyjUmQjOCtbS0qKOjQx0dHQ8FMdsOVNevX6/e3l7t2rVL3d3dKi4u1pkzZx66QJxN9u3bJ0kqLy8fsP7gwYOqrKzM/EAjGKdOAMxl11VDAKMSoQFgjtAAMEdoAJgjNADMERoA5ggNAHOEBoA5QgPAHKEBYI7QADD3v6leF/E7foTsAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "-1.0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "play(True)[-1]"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "第19章-目标导向的强化学习.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python [conda env:cuda117]",
   "language": "python",
   "name": "conda-env-cuda117-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
